# split batch大小
BATCH_SIZE_SPLIT = 200
# MODEL batch大小
BATCH_SIZE_MODEL = 50

# es batch的大小
BATCH_SIZE_ES = 100000

# 最多启动多少进程
max_workers = 8

# 需要转换为向量的字段
to_vector_properties = ["news_title_zh", "news_summary"]

# 每几个batch输出屏幕
INFO_UPDATE_FACTOR = 10
# Replace the model name with 'msmarco-distilbert-base-dot-prod-v3' to map sentences to
# a 768 dimensional dense vector space.
# 模型
MODEL_NAME = 'all-MiniLM-L6-v2'
# MODEL_NAME = 'msmarco-distilbert-base-dot-prod-v3'




ID = "id"
NEWS_SOURCE = "news_source"
NEWS_WEBSITE = "news_website"
NEWS_WEBSITE_TYPE = "news_website_type"
NEWS_URL = "news_url"
NEWS_TITLE = "news_title"
NEWS_CONTENT = "news_content"
NEWS_AUTHOR = "news_author"
NEWS_MEDIA_NAME = "news_media_name"
NEWS_PUBLICTIME = "news_publictime"
NEWS_PUBLICDATE = "news_publicdate"
NEWS_LANGUAGE = "news_language"
NEWS_TITLE_ZH = "news_title_zh"
NEWS_CONTENT_ZH = "news_content_zh"
NEWS_KEYWORDS = "news_keywords"
NEWS_SUMMARY = "news_summary"
NEWS_ENTITIES = "news_entities"
NEWS_ENTITIES_PERSON = "news_entities_person"
NEWS_ENTITIES_PLACE = "news_entities_place"
NEWS_ENTITIES_ORGANIZATION = "news_entities_organization"
NEWS_TYPE = "news_type"
CRAWL_TIME = "crawl_time"
DOMAIN = "domain"
TITLE_CONTENT_VECTOR = "title_content_vector"
TITLE_KEYWORDS = "title_keywords"
mapping = {
"mappings": {
            "properties": {
                "crawl_time": {
                    "type": "date",
                    "format": "yyyy-MM-dd HH:mm:ss||epoch_second"
                },
                "domain": {
                    "type": "keyword"
                },
                "id": {
                    "type": "keyword"
                },
                "news_author": {
                    "type": "keyword"
                },
                "news_cluster_id": {
                    "type": "keyword"
                },
                "news_content": {
                    "type": "text"
                },
                "news_content_zh": {
                    "type": "text",
                    # "fields": {
                    #     "news_content_zh_keyword": {
                    #         "type": "wildcard"
                    #     }
                    # },
                    "analyzer": "ik_max_word"
                },
                "news_entities": {
                    "type": "keyword"
                },
                "news_entities_organization": {
                    "type": "keyword"
                },
                "news_entities_person": {
                    "type": "keyword"
                },
                "news_entities_place": {
                    "type": "keyword"
                },
                "news_keywords": {
                    "type": "keyword"
                },
                "news_language": {
                    "type": "keyword"
                },
                "news_media_name": {
                    "type": "keyword"
                },
                "news_publicdate": {
                    "type": "date",
                    "format": "yyyy-MM-dd||epoch_second"
                },
                "news_publictime": {
                    "type": "date",
                    "format": "yyyy-MM-dd HH:mm:ss||epoch_second"
                },
                "news_source": {
                    "type": "integer"
                },
                "news_summary": {
                    "type": "text",
                    "analyzer": "ik_max_word"
                },
                "news_title": {
                    "type": "text"
                },
                "news_title_zh": {
                    "type": "text",
                    # "fields": {
                    #     "news_title_zh_keyword": {
                    #         "type": "wildcard"
                    #     }
                    # },
                    "analyzer": "ik_max_word"
                },
                "news_type": {
                    "type": "keyword"
                },
                "news_url": {
                    "type": "keyword"
                },
                "news_website": {
                    "type": "keyword"
                },
                "news_website_type": {
                    "type": "keyword"
                }
            }
        }
}


jsons = mapping["mappings"]["properties"]


# 所有索引属性名称
news_property = (list(mapping["mappings"]["properties"].keys()))

# vector的名字
vector_name = "title_content_vector"

########################## 必须修改开始 ##################################

# 用户名
ELASTIC_USERNAME = "elastic"
# 用户密码
ELASTIC_PASSWORD = "iie123$%^"

MODEL_PATH = r"/tmp/es2/model"
# 集群地址
ELASTIC_ADDRESS = ["http://10.26.32.92:9222"]
# 索引名称
INDEX_NAME = "test_news_v1"
# 数据存放根目录
DATA_FILE_PATH = r"/mnt/sdn/zjh/es/test1"
# 分片大小
SHARDS = 1
# 副本大小
NUM_REPLICA = 0
MAX_RESULT_WINDOW = 100000

########################## 必须修改结束 ##################################

# 配置要更新的设置
settings = {
	"settings": {
		"number_of_shards": SHARDS,
		"number_of_replicas": NUM_REPLICA,
		"max_result_window": MAX_RESULT_WINDOW
	}
}